{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "import re\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import csv\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser= webdriver.Chrome()#声明浏览器\n",
    "url = 'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/forumdetail?pid=1318879010'\n",
    "browser.get(url)#打开浏览器预设网址\n",
    "time.sleep(3)\n",
    "browser.find_element_by_xpath(r'/html/body/div[5]/div[2]/div[1]/div/div[3]/div/div[2]/div/div[2]/div[2]/div/span').click()\n",
    "time.sleep(3)\n",
    "#/html/body/div[14]/div[2]/div/div/div/div/div[2]/span\n",
    "browser.find_element_by_xpath(r'/html/body/div[14]/div[2]/div/div/div/div/div[2]/span').click()\n",
    "time.sleep(8)\n",
    "#微信登入\n",
    "browser.find_element_by_xpath(r'/html/body/div[14]/div[2]/div/div/div/div/div/div[1]/div/div[2]/div[1]/a[2]/span').click()\n",
    "time.sleep(10)\n",
    "#点击参加我的课程\n",
    "browser.find_element_by_xpath(r'/html/body/div[5]/div[2]/div[1]/div/div[3]/div/div[2]/div/div[2]/div[2]/div/span').click()\n",
    "time.sleep(8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#输入每个章节的URL地址\n",
    "#输入每个章节的URL地址\n",
    "urls = ['https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1238139963&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1238139935&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1238139948&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1238139955&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1238139984&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1238139972&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1239464404&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1238139996&sm=1',\n",
    "\n",
    "'https://www.icourse163.org/spoc/learn/CUEB-1460898185?tid=1461659481#/learn/content?type=detail&id=1239486337&sm=1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#进入每个讨论区章节\n",
    "for url in urls:\n",
    "    browser.get(url)\n",
    "    browser.refresh()\n",
    "    time.sleep(5)\n",
    "    #寻找页数的xpath路径\n",
    "    myPage=browser.find_elements_by_xpath(r'//*[@id=\"courseLearn-inner-box\"]/div/div/div[3]/div[1]/div[2]/div/div/div[4]/div/div[1]/div[2]')\n",
    "    myPagePatten=\"style=\\\"\\\">(\\d*?)</a>\"#内容页匹配符\n",
    "    Pages=re.findall(myPagePatten,myPage[0].get_attribute('innerHTML'))#从源代码中进行匹配\n",
    "    Page=int(Pages[-1])-1#获得页数\n",
    "\n",
    "    for i in range(Page): #页数遍历       \n",
    "        time.sleep(3)\n",
    "        #获取内容页xpath路径\n",
    "        contents_main=browser.find_elements_by_xpath(\"//div[contains(@class,'j-list')]//div[contains(@class,'m-data-lists')]/div[contains(@class,'f-pr')]/div[contains(@class,'m-detailInfoItem f-pr') and contains(@style,'100')]\")\n",
    "        page_nexts=browser.find_elements_by_link_text('下一页')#下一页\n",
    "        page_next=page_nexts[len(page_nexts)-1]\n",
    "        myMainPatten=\"\\+\\+(\\d*?)\\-\\-(.*?)xx(\\d*?)\\=\\=([\\s\\S]*)举报\"#学号姓名匹配符\n",
    "        myCommentPagePatten=\"style=\\\"\\\">(\\d*?)</a>\"#评论匹配符\n",
    "        mySignPatten=\"\\+\\+(\\d*?)\\-\\-(.*?)xx(\\d*?)\\=\\=\"#签到匹配符\n",
    "        for content_main in contents_main:\n",
    "            content_main_text=content_main.text\n",
    "            if \"签到\" not in content_main_text:\n",
    "                content_main_data=re.findall(myMainPatten,content_main_text)     #获取问答区内容\n",
    "                if not os.path.exists('content_main_result.csv'): \n",
    "                    headers = ['Number', 'name','position','content'] #设置表头\n",
    "                    with open('content_main_result.csv', 'w', newline = '', encoding = 'utf8') as f:  \n",
    "                        f_csv = csv.writer(f)  #写入文件\n",
    "                        f_csv.writerow(headers)  \n",
    "                with open('content_main_result.csv', 'a', newline = '', encoding = 'utf8') as f:\n",
    "                    f_csv = csv.writer(f)  \n",
    "                    for i in range(len(content_main_data)):\n",
    "                        f_csv.writerow(content_main_data[i])\n",
    "                father_content=content_main.find_element_by_xpath('..')#第一内页\n",
    "                father_content_text=father_content.get_attribute('innerHTML')#获取源码\n",
    "                if 'm-commentWrapper' in father_content_text:                      #获取评论区内容\n",
    "                    comment_content=content_main.find_element_by_xpath(\"../div[contains(@class,'m-commentWrapper f-pr')]\")\n",
    "                    comment_content_text=comment_content.text\n",
    "                    comment_content_data=re.findall(myMainPatten,comment_content_text) \n",
    "                    if not os.path.exists('content_comment_result.csv'): \n",
    "                        headers = ['Number', 'name','position','content'] \n",
    "                        with open('content_comment_result.csv', 'w', newline = '', encoding = 'utf8') as f:  \n",
    "                            f_csv = csv.writer(f)  \n",
    "                            f_csv.writerow(headers)  \n",
    "                    with open('content_comment_result.csv', 'a', newline = '', encoding = 'utf8') as f:\n",
    "                        f_csv = csv.writer(f)  \n",
    "                        for i in range(len(comment_content_data)):\n",
    "                            f_csv.writerow(comment_content_data[i])\n",
    "            else:\n",
    "                father_content=content_main.find_element_by_xpath('..')\n",
    "                father_content_text=father_content.get_attribute('innerHTML')\n",
    "                if 'm-commentWrapper' in father_content_text:                            #获取签到列表\n",
    "                    sign_content=content_main.find_element_by_xpath(\"../div[contains(@class,'m-commentWrapper f-pr')]\")  \n",
    "                    comment_page_info=sign_content.get_attribute('innerHTML')#获取签到区源码\n",
    "                    comment_pages=re.findall(myCommentPagePatten,comment_page_info)#进行匹配\n",
    "                    if comment_pages:\n",
    "                        comment_page=int(comment_pages[-1])\n",
    "                        for i in range(comment_page):\n",
    "                            time.sleep(1)\n",
    "                            sign_content_text=sign_content.text\n",
    "                            sign_content_text_filter=re.findall(mySignPatten,sign_content_text)                        \n",
    "                            if not os.path.exists('content_sign_result.csv'): \n",
    "                                headers = ['Number', 'name','position'] \n",
    "                                with open('content_sign_result.csv', 'w', newline = '', encoding = 'utf8') as f:  \n",
    "                                    f_csv = csv.writer(f)  \n",
    "                                    f_csv.writerow(headers)  \n",
    "                            with open('content_sign_result.csv', 'a', newline = '', encoding = 'utf8') as f:\n",
    "                                f_csv = csv.writer(f)  \n",
    "                                for i in range(len(sign_content_text_filter)):\n",
    "                                    f_csv.writerow(sign_content_text_filter[i])\n",
    "                            sign_content.find_element_by_link_text('下一页').click()\n",
    "                    else:\n",
    "                        sign_content_text=sign_content.text\n",
    "                        sign_content_text_filter=re.findall(mySignPatten,sign_content_text)\n",
    "                        if not os.path.exists('content_sign_result.csv'): \n",
    "                            headers = ['Number', 'name','position'] \n",
    "                            with open('content_sign_result.csv', 'w', newline = '', encoding = 'utf8') as f:  \n",
    "                                f_csv = csv.writer(f)  \n",
    "                                f_csv.writerow(headers)  \n",
    "                        with open('content_sign_result.csv', 'a', newline = '', encoding = 'utf8') as f:\n",
    "                            f_csv = csv.writer(f)  \n",
    "                            for i in range(len(sign_content_text_filter)):\n",
    "                                f_csv.writerow(sign_content_text_filter[i])\n",
    "\n",
    "        page_next.click()\n",
    "    time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
